import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;

public class ArticleStandard {

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		ArrayList<String> dictionary = new ArrayList<String>();
		String fileDicName = "dictionary.txt";
		try {
			File fileDir = new File(fileDicName);

			BufferedReader in = new BufferedReader(new InputStreamReader(
					new FileInputStream(fileDir), "UTF8"));

			String str;			

			while ((str = in.readLine()) != null) {
				String[] temp = str.split("");
//				str = str.trim();
//				for(int i = 0; i < temp.length; i++){
//					System.out.print(temp[i]);
//					str += temp[i];
//				}
//				System.out.print(temp.length);
//				System.out.println("\n");
//				str = str.substring(temp.length/2);
				dictionary.add(str);
//				System.out.println(str);
			}

			in.close();
		} catch (UnsupportedEncodingException e) {
			System.out.println(e.getMessage());
		} catch (IOException e) {
			System.out.println(e.getMessage());
		} catch (Exception e) {
			System.out.println(e.getMessage());
		}
		java.util.Collections.sort(dictionary);
//		if(checkContain(dictionary, "hiểu")){
//			System.out.println("có hiểu");
//		} else {
//			System.out.println("không có hiểu");
//		}
//		for(int i = 0; i < dictionary.size(); i++){
//			System.out.println(dictionary.get(i));
//		}
		// ---------------------------------------------

//		String[] test = {"hiểu", "anh"};
//		for(int i = 0; i < test.length; i++){
//			if(test[i].equalsIgnoreCase("hiểu")){
//				System.out.println("có hiểu");
//			} else {
//				System.out.println("không có hiểu");
//			}
//		}
		
		ArrayList<String> docs = new ArrayList<String>();

		String fileWithoutPuctuation = "";
		int numOfArticle = 15000;
		// String regex = "[0-9]+";

		for (int i = 0; i < numOfArticle; i++) {
			String fileName = "";
			fileName = "C:\\Users\\09520_000\\Desktop\\thethao\\thethao (" + (i + 1) + ").txt";
			System.out.println(fileName);

			fileWithoutPuctuation = readFile(fileName);
//			System.out.println("fileWithoutPuctuation 1 :  " + fileWithoutPuctuation);
			// remove special characters
			fileWithoutPuctuation = fileWithoutPuctuation.replaceAll(
					"[^\\p{L}\\p{Nd}]", " ");
//			System.out.println("fileWithoutPuctuation 2 :  " + fileWithoutPuctuation);
			//replace multiple blanks with only one blank
			fileWithoutPuctuation = fileWithoutPuctuation.replaceAll("\\s+", " "); 
//			System.out.println("fileWithoutPuctuation 3 :  " + fileWithoutPuctuation);
			String[] articleWords = fileWithoutPuctuation.split(" ");
//			System.out.println("articleWords  :  " + articleWords);
			String tempArticle = "";

			for (int j = 0; j < articleWords.length; j++) {
//				System.out.println("articleWords[j]  :  " + articleWords[j]);
//				System.out.println("dictionary[j]  :  " + dictionary.get(j));
				if(checkContain(dictionary, articleWords[j])){
					tempArticle += articleWords[j] + " ";					
				}
//				if (dictionary.contains(articleWords[j])) {
//					tempArticle += articleWords[j] + " ";
//					System.out.println("articleWords[j]  :  " + articleWords[j]);
//				}
			}
			docs.add(tempArticle);

		}

		for(int i = 0; i < docs.size(); i++){
			writeFile("C:\\Users\\09520_000\\Desktop\\thethao2\\thethao (" + (i + 1) + ").txt", docs.get(i));
		}
		// writeFile(vectorFile, writeString);

	}
	
	static boolean checkContain(ArrayList<String> dictionary, String word){
		for(int i = 0; i < dictionary.size(); i++){
//			System.out.println("dictionary.get(i)  :  " + dictionary.get(i));
//			System.out.println("word               :  " + word);
//			if(dictionary.get(i).equalsIgnoreCase("hiểu")){
//				System.out.println("dictionary.get(i)  :  " + dictionary.get(i));
//				break;
//			}
//			if(word.trim().equalsIgnoreCase("hiểu")){
//				System.out.println("word               :  " + word);
//				break;
//			}
			if(dictionary.get(i).equalsIgnoreCase(word.trim())){
				return true;
			}
		}
		
		return false;
	}

	static String readFile(String fileName){
		String temp = "";
		File fileDir = new File(fileName);

		BufferedReader in;
		try {
			in = new BufferedReader(new InputStreamReader(
					new FileInputStream(fileDir), "UTF8"));
			String str;

			while ((str = in.readLine()) != null) {
				temp +=str;
			}

			in.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}		

		return temp;
	}
	
	static String readFile2(String fileName) {
		String temp = "";
		try (BufferedReader br = new BufferedReader(new FileReader(fileName))) {
			String sCurrentLine;
			while ((sCurrentLine = br.readLine()) != null) {
				temp += sCurrentLine;
			}
		} catch (IOException e) {
			e.printStackTrace();
		}

		return temp;
	}

	static void writeFile(String fileName, String writeString) {
		try {
			File fileDir = new File(fileName);
			BufferedWriter out = new BufferedWriter(new OutputStreamWriter(
					new FileOutputStream(fileDir), "UTF8"));

			out.write(writeString);

			out.flush();
			out.close();
		} catch (UnsupportedEncodingException e) {
			System.out.println(e.getMessage());
		} catch (IOException e) {
			System.out.println(e.getMessage());
		} catch (Exception e) {
			System.out.println(e.getMessage());
		}
	}
}
